#!/bin/bash
#设置任务异常退出
set -eo pipefail
dt='{{ execution_date | date_add(1) | cst_ds }}'
name='中心滞留明细'
# 在退出时清理文件
trap "rm -f /tmp/${name}_${dt}.zip /tmp/${name}_${dt}.csv && echo 'removed tmp files'" EXIT
# 创建txt文件
touch /tmp/${name}_${dt}.txt
#将中文字段名输入文件顶部
headline="运单号,最后一次到件时间,扫描类型,代理区编码,代理区名称,中心code,中心,一段码,二段码,派件网点,省,城市,区,是否退转件,是否是疫情件,问题件类型code,问题件类型名称,是否是拦截件,系统时间,小时差,小时类型,时间分区"
#将查询结果追加到文件中
{
  spark-sql --driver-memory 16G --num-executors 10 --executor-memory 8g --executor-cores 4  --conf spark.executor.memoryOverhead=4g --conf spark.driver.maxResultSize=3g -S -e "
select waybill_no
, scan_time
,'中心到件扫描' as scan_type 
, site_agent_code
, site_agent_name
, scan_site_code
, scan_site
, nvl(first_code,'--') as first_code
, nvl(second_code,'--') as second_code
, nvl(dispatch_code,'--') as dispatch_code
, nvl(provider_desc,'--') as provider_desc
, nvl(city_desc,'--') as city_desc
, nvl(area_desc,'--') as area_desc
, nvl(is_reback,'--') as is_reback
, nvl(is_yiqing_problem,'--') as is_yiqing_problem
, nvl(second_level_type_code,'--') as second_level_type_code
, nvl(second_level_type_name,'--') as second_level_type_name
, nvl(is_intercepte,'--') as is_intercepte
, nvl(sys_time,'--') as sys_time
, hours
, hour_type
, dt
from bidefault.center_yiqing_tmp 
where dt between date_add('{{ execution_date | cst_ds }}',-8) and date_add('{{ execution_date | cst_ds }}',-1)
and hours>=48;
 " >>/tmp/${name}_${dt}.txt
}

# 替换分隔符，并输出csv
cat /tmp/${name}_${dt}.txt | tr -s '\t' ',' > /tmp/${name}_${dt}.csv
#分割文件为100w一个
/usr/local/python3/bin/python3  /usr/local/airflow/dags/utils/operators/file_split.py /tmp/${name}_${dt}.csv ${headline}
#压缩
zip -jrm /tmp/${name}_${dt}.zip /tmp/${name}_${dt}/*
# 压缩并删除 csv 文件
#zip -jrm /tmp/${name}_${dt}.zip /tmp/${name}_${dt}.csv
# 上传文件
/usr/local/python3/bin/python3  /usr/local/airflow/dags/utils/operators/upload_oss.py ${name}/${name}_${dt}.zip /tmp/${name}_${dt}.zip
# zip 文件将在进程退出后被自动清理

#声明文件大小
data_size=$(ls -lh /tmp/${name}_${dt}.zip | awk '{print $5}')
#打印文件信息
Message="${name}_${dt}.zip 上传成功 大小: $data_size"
#配置钉钉告警
curl 'https://oapi.dingtalk.com/robot/send?access_token=816a8f35fd84432ffa6bff9d7714abc322628bb4a8ded86908550d2335aba613' \
   -H 'Content-Type: application/json' \
   -d "
  {\"msgtype\": \"text\",
    \"text\": {
        \"content\": \"监控内容:$Message\"
     }
  }" || true
rm -f  /tmp/${name}_${dt}.txt